# @Time : 2024-11-05 11:06
# @Author : Fioman 
# @Phone : 13149920693
"""
通过继承thread类来进行多线程网络爬虫
"""
import time
from threading import Thread

import requests


class WebScraper(Thread):
    def __init__(self, name, url):
        super().__init__(name=name)
        self.url = url

    def run(self):
        print(f"Starting to scrape: {self.url}")
        try:
            response = requests.get(self.url, timeout=5)
            response.raise_for_status()  # 如果状态码不是200,会抛出长
            print(f"Success: {self.url} - Length: {len(response.text)}")
        except Exception as e:
            print(f"Failed to scrape {self.url}: {str(e)}")


if __name__ == '__main__':
    urls = [
        "https://www.example.com",
        "https://www.python.org",
        "https://www.baidu.com",
        "https://www.openai.com",
        "https://www.github.com"
    ]
    threads = []
    startTime = time.time()
    for url in urls:
        scaper = WebScraper(url, url)
        threads.append(scaper)
        scaper.start()

    for thread in threads:
        thread.join()

    endTime = time.time()
    print(f"Total time: {endTime - startTime}")
